package com.karateca.columbiaimports;

import com.karateca.columbiaimports.http.HttpDownloader;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

/**
 * 
 * This class contains different methods to download and parse the different
 * set of pages that conform an import. An import has the 4 pages:
 * <ul>
 * <li>DUA - This page contains a link to the DUA report and a link to the 
 * format B (DUA B). No data is collected from this page.</li>
 * <li>DUA report - contains all the information about the import along with
 * the series data.</li>
 * <li>Intermidiate DUA B page - contains a list of links to the DUA B format.</li>
 * <li>DUA B page - contains data about the DUA B format.</li>
 * </ul>
 * 
 * @author Andres Dominguez
 */
public class ImportDownloader
{
    /** Apache log. */
    private static final Log log = LogFactory.getLog(ImportDownloader.class);
    /** Customs codes */
    private static final String[] CUSTOMS =
    {
        "019", "046", "055", "082", "091", "118", "127", "145", "154",
        "163", "172", "181", "190", "217", "226", "235", "262", "280"
    };
    /** Server url */
    public static final String SERVER_URL = "http://www.aduanet.gob.pe";
    /** Start search date */
    private String startDate;
    /** End search date */
    private String endDate;
    /** Initialized downloader used to get the data from the server */
    private HttpDownloader downloader;
    /** null value constant */
    public static final String NULL_VALUE = "null";

    /**
     * Creates a new instance of ImportDownloader.
     * @param startDate start search date.
     * @param endDate en search date.
     * @param httpDownloader initialized downloader used to get the data from the server
     */
    public ImportDownloader(String startDate, String endDate, HttpDownloader httpDownloader)
    {
        this.startDate = startDate;
        this.endDate = endDate;
        this.downloader = httpDownloader;
    }

    /**
     * Parse all the data in a DUA B report page.
     * @param duaBPage page containing the DUA.
     * @param link DUA url, used to print warning.
     * @return a map containing column names as keys and their corresponding values.
     */
    public static Map<String, String> parseDuaReportB(String duaBPage, String link)
    {
        // Replace HTML spaces by spaces
        duaBPage = duaBPage.replaceAll("&nbsp;", " ");

        // Formats are different before and after 2001, test the type of report
        if (duaBPage.contains("UNICA DE ADUANAS (B)"))
        {
            // After jan 1, 2001
            return parseNewDuaReportB(duaBPage, link);
        }
        else
        {
            // before dec 31, 2000
            return parseOldDuaReportB(duaBPage, link);
        }
    }

    /**
     * Parse all the data in a DUA B report page. These reports are from before
     * dec 31, 2000.
     * @param duaBPage page containing the DUA.
     * @param link DUA url, used to print warning.
     * @return a map containing column names as keys and their corresponding values.
     */
    private static Map<String, String> parseOldDuaReportB(String duaBPage, String link)
    {
        String[] rows;
        Map<String, String> duaMap;
        List<String> columns;
        String regEx;
        int i;

        duaMap = new TreeMap<String, String>();
        regEx = "(<FONT SIZE=2><P( ALIGN='CENTER')?>)(.*?)(</FONT></TD>)";
        rows = duaBPage.split("<TR><TD WIDTH");

        // Check if page is empty
        if (rows.length == 1)
        {
            log.info("Found empty DUA B report in [" + link + "]");
            return new EmptyMap(NULL_VALUE);
        }

        // Row 10, 3.1 and 3.2
        columns = findPatterns(rows[8], regEx, 3);
        i = 0;
        duaMap.put("3.1 Nombre / Razon social", columns.get(i++));
        duaMap.put("3.2 Condicion", columns.get(i++));

        // Row 12 3.3 to 3.7
        columns = findPatterns(rows[10], regEx, 3);
        i = 0;
        duaMap.put("3.3 Direccion", columns.get(i++));
        duaMap.put("3.3 Codigo", NULL_VALUE);
        duaMap.put("3.4 Ciudad", columns.get(i++));
        duaMap.put("3.5 Pais", columns.get(i++));
        duaMap.put("3.6 Fax", columns.get(i++));
        duaMap.put("3.7 Telefono", columns.get(i++));
        duaMap.put("3.8 Pagina Web", NULL_VALUE);
        duaMap.put("3.9 E-mail", NULL_VALUE);

        // Set all section 7 to null
        for (i = 1; i <= 15; i++)
        {
            duaMap.put("7." + i, NULL_VALUE);
        }

        return duaMap;
    }

    /**
     * Parse all the data in a DUA B report page. These reports are from jan 1,
     * 2001 and later.
     * @param duaBPage page containing the DUA.
     * @param link DUA url, used to print warning.
     * @return a map containing column names as keys and their corresponding values.
     */
    private static Map<String, String> parseNewDuaReportB(String duaBPage, String link)
    {
        String[] tables;
        List<String> values;
        Map<String, String> duaMap;
        String regEx;
        int i;

        duaMap = new TreeMap<String, String>();
        // Split tables
        tables = duaBPage.split("<table border='1' width='100%'>");

        // Get PROVEEDOR data
        // Search the following pattern:
        // <font face='MS Sans Serif' size='1'>POULTEN & GRAF GMBH</font>
        regEx = "(<font face='MS Sans Serif' size='1'>)(.*?)(</font>)";
        values = findPatterns(tables[4], regEx, 2);
        // Test if it found 10 matches or a different count
        if (values.size() != 10)
        {
            // If the count is 9 then "3.2 Condicion" is missing. Check the number
            // of font tags (42)
            if ((values.size() == 9) && (findPatterns(tables[4], "font", 0).size() == 42))
            {
                log.warn("DUA has missing element [3.2 Condicion], adding null. link: " + link);
                values.add(1, NULL_VALUE);
            }
            else
            {
                String errMsg = "Error parsing Reporte DUA B, found " + values.size() + ", expected: 10";
                log.error(errMsg);
                throw new RuntimeException(errMsg);
            }
        }
        i = 0;
        duaMap.put("3.1 Nombre / Razon social", values.get(i++));
        duaMap.put("3.2 Condicion", values.get(i++));
        duaMap.put("3.3 Direccion", values.get(i++));
        duaMap.put("3.3 Codigo", values.get(i++));
        duaMap.put("3.4 Ciudad", values.get(i++));
        duaMap.put("3.5 Pais", values.get(i++));
        duaMap.put("3.6 Fax", values.get(i++));
        duaMap.put("3.7 Telefono", values.get(i++));
        duaMap.put("3.8 Pagina Web", values.get(i++));
        duaMap.put("3.9 E-mail", values.get(i++));

        // Get CONDICIONES DE LA TRANSACCION data
        // Search the following pattern:
        // <input name='sel' type='checkbox' checked disabled>
        regEx = "(<input name='sel' type='checkbox' checked disabled>)|(<input name='sel' type='checkbox' disabled>)";
        values = findPatterns(tables[8], regEx, 0);
        // Test if it found 14 matches
        if ((values.size() != 14) && (values.size() != 15))
        {
            String errMsg = "Error parsing Reporte DUA B, found " + values.size() +
                    " checkboxes, expected: 14 or 15";
            log.error(errMsg);
            throw new RuntimeException(errMsg);
        }
        // Checked values have a value of Yes, others have a No
        i = 0;
        duaMap.put("7.1", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.8", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.2", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.9", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.3", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.10", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.4", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.11", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.5", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.12", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.6", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.13", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.7", values.get(i++).contains("checked") ? "Yes" : "No");
        duaMap.put("7.14", values.get(i++).contains("checked") ? "Yes" : "No");
        // Check if there are 15 checkboxes
        if (values.size() == 15)
        {
            duaMap.put("7.15", values.get(i++).contains("checked") ? "Yes" : "No");
        }
        else
        {
            duaMap.put("7.15", "No");
        }

        return duaMap;
    }

    /**
     * Find a number of patterns in a block of text and return the matches as
     * a list of string.
     * @param text text to be searched.
     * @param regEx regular expression to match.
     * @param patternGroup group number.
     * @return a list of matches.
     */
    private static List<String> findPatterns(String text, String regEx, int patternGroup)
    {
        List<String> results;
        Pattern pattern;
        Matcher matcher;

        results = new ArrayList<String>();
        pattern = Pattern.compile(regEx);
        matcher = pattern.matcher(text);
        while (matcher.find())
        {
            results.add(matcher.group(patternGroup).trim());
        }
        return results;
    }

    /**
     * Find link to final DUA report in the intermidiate page.
     * @param page HTML page containing the link.
     * @return the link to the DUA B.
     * @throws java.lang.RuntimeException when the link cannot be found.
     */
    public static String findDuaBFinalLink(String page)
    {
        String regEx = "";

        // There are two different patterns depending of the date
        if (page.contains("SgCDUIBdua"))
        {
            // 2001 and later
            // <a href="/servlet/SgCDUIBdua?codiaduan=118&anoprese=05&numecorre=000087&
            // numesecuf=1&pcodi_regi=10" target = '_top'>
            regEx = "(/servlet/SgCDUIBdua\\?[a-zA-z0-9&= ]*)(\" target = '_top'>)";
        }
        else if (page.contains("SgCDUIB"))
        {
            // before 2001
            // <a href="/servlet/SgCDUIB?codiaduan=118&anoprese=99&numecorre=004534&
            // numesecuf=1" target = '_top'>
            regEx = "(/servlet/SgCDUIB\\?[a-zA-z0-9&= ]*)(\" target = '_top'>)";
        }
        else
        {
            String errMsg = "Could not find DUA B link pattern in page";
            log.error(errMsg);
            throw new RuntimeException(errMsg);
        }

        return SERVER_URL + findLinkPattern(page, regEx, 1);
    }

    /**
     * Find the DUA report link.
     * @param page HTML page containing the link.
     * @return the link to the DUA report.
     * @throws java.lang.RuntimeException when the link cannot be found.
     */
    public static String findDuaReportLink(String page)
    {
        // search the following pattern:
        // <a href="/ol-ad-itconsultadua/RPSGDui?ndui=1180510000122">
        String regEx = "(/ol-ad-itconsultadua/RPSGDui\\?[a-zA-z0-9&= ]*)(\">)";
        return SERVER_URL + findLinkPattern(page, regEx, 1);
    }

    /**
     * Find the DUA B link.
     * @param page HTML page containing the link.
     * @return the link to the dua report.
     * @throws java.lang.RuntimeException when the link cannot be found.
     */
    public static String findDuaBLink(String page)
    {
        // search the following pattern:
        // <a href="/servlet/SGDValor?codiaduan=118&numecorre=000122&anoprese=2005
        // &tcodigo=4&pcodi_regi=10">
        String regEx = "(/servlet/SGDValor\\?[a-zA-z0-9&= ]*)(\">)";
        return SERVER_URL + findLinkPattern(page, regEx, 1);
    }

    /**
     * Find a link given a regular expression and a group.
     * @param page HTML page containing the link.
     * @param regEx regular expression to search the link.
     * @param patternGroup group number within regular expression.
     * @return the link that matches the regular expression.
     * @throws java.lang.RuntimeException when the link cannot be found.
     */
    private static String findLinkPattern(String page, String regEx, int patternGroup)
    {
        String link;
        Pattern pattern;
        Matcher matcher;

        pattern = Pattern.compile(regEx);
        matcher = pattern.matcher(page);

        if (matcher.find())
        {
            link = matcher.group(patternGroup);
        }
        else
        {
            String errMsg = "Cannot find link in page " + page;
            throw new RuntimeException(errMsg);
        }
        return link;
    }

    /**
     * Parse a DUA report along with its series. The first element in the
     * resulting list contains the report header. The following elements are
     * the indivifual series.
     * @param duaReport page containing the DUA report type A.
     * @param link url of the import. The link is printed in case of warning.
     * @return a list of maps containing column names as keys and their
     * corresponding values.
     */
    public static List<Map<String, String>> parseDuaReport(String duaReport, String link)
    {
        List<Map<String, String>> series;
        String[] seriesBlocks;
        int seriesCount;
        String block;

        series = new ArrayList<Map<String, String>>();

        // Get the number of series
        seriesCount = parseSeriesCount(duaReport);
        if (seriesCount < 1)
        {
            String errMsg = "Error parsing series, couldn't find number of series";
            log.error(errMsg);
            throw new RuntimeException(errMsg);
        }

        // Split page into different series blocks and check if count matches
        seriesBlocks = duaReport.split("<tr><td><font size=1>[0-9]{1,10}</font></td>");
        if ((seriesBlocks.length - 1) != seriesCount)
        {
            log.warn("Number of series found does not match series count. " +
                    "Parsed value: " + seriesCount + ", blocks found: " +
                    (seriesBlocks.length - 1) + ", link " + link);
        }

        log.debug("Found " + seriesCount + " series");

        // First element is the header
        series.add(parseReportDuaHeader(seriesBlocks[0]));

        // Remove code from last block
        block = seriesBlocks[seriesBlocks.length - 1];
        block = block.substring(0, block.indexOf("</font></td></tr></table>"));
        seriesBlocks[seriesBlocks.length - 1] = block;

        // Remaining elements are individual series
        for (int i = 1; i < seriesBlocks.length; i++)
        {
            series.add(parseIndivisualSeries(seriesBlocks[i], i));
        }

        return series;
    }

    /**
     * Parse the DUA report type A header.
     * @param duaBlock HTML containing the header data.
     * @return a map containing column names as keys and values.
     */
    private static Map<String, String> parseReportDuaHeader(String duaHeader)
    {
        Map duaMap;
        String[] rows;
        List<String> columns;

        duaMap = new TreeMap();
        // Replace HTML spaces by spaces
        duaHeader = duaHeader.replaceAll("&nbsp;", " ");
        // Split rows. Some rows are not closed properly (<tr><tr>)
        rows = duaHeader.split("(</tr><tr>)|(<tr><tr>)");

        // Row 2: 2 (DECLARACION), 4 (FECHA NUMERACION), 6 (SUJETO A)
        columns = getRowColumns(rows[1]);
        duaMap.put("DECLARACION", columns.get(1));
        duaMap.put("FECHA NUMERACION", columns.get(3));
        duaMap.put("SUJETO A", columns.get(5));

        // Row 4: 2 (ADUANA), 3 (CODIGO ADUANA), 5 (DECLARANTE), 6 (CODIGO DECLARANTE)        
        columns = getRowColumns(rows[3]);
        duaMap.put("1.1 ADUANA", columns.get(1));
        duaMap.put("1.1 CODIGO ADUANA", columns.get(2));
        duaMap.put("1.2 DECLARANTE", columns.get(4));
        duaMap.put("1.2 CODIGO DECLARANTE", columns.get(5));

        // Row 5: 2 (CODIGO IMPORTADOR), 3 (IMPORTADOR), 5 (DIRECCION)
        columns = getRowColumns(rows[4]);
        duaMap.put("1.3 CODIGO IMPORTADOR", columns.get(1));
        duaMap.put("1.3 IMPORTADOR", columns.get(2));
        if (columns.size() < 5)
        {
            log.warn("Missing field [1.5 DIRECCION] in DUA Header " + duaMap.get("DECLARACION"));
            duaMap.put("1.5 DIRECCION", NULL_VALUE);
        }
        else
        {
            duaMap.put("1.5 DIRECCION", columns.get(4));
        }

        // Row 7: 2 (3.1. MONEDA), 4 (3.2.- FORMA DE PAGO), 6 (3.3. PLAZO)
        columns = getRowColumns(rows[6]);
        duaMap.put("3.1 MONEDA", columns.get(1));
        duaMap.put("3.2 FORMA DE PAGO", columns.get(3));
        duaMap.put("3.3 PLAZO", columns.get(5));

        // Row 8: 2 (3.4. ENT.FINANC.), 4 (3.2.- FEC.CARTA CRED),
        // 6 (3.6. EXONERACION C.I.)
        columns = getRowColumns(rows[7]);
        duaMap.put("3.4 ENT.FINANC.", columns.get(1));
        duaMap.put("3.5 FEC.CARTA CRED", columns.get(3));
        duaMap.put("3.6 EXONERACION C.I.", columns.get(5));

        // Row 10: 2 (4.1. MANIFIESTO :), 3 (4.2 MODO :  0), 4 (4.3. EMPRESA),
        // 6 (4.4. FEC.LLEGADA)
        columns = getRowColumns(rows[9]);
        duaMap.put("4.1 MANIFIESTO", columns.get(1));
        duaMap.put("4.2 MODO", columns.get(2).substring(10).trim());
        duaMap.put("4.3 EMPRESA", columns.get(3).substring(14).trim());
        duaMap.put("4.4. FEC.LLEGADA", columns.get(5));

        // Row 12: 2 (5.1. TERMINAL), (5.1. CODIGO TERMINAL)
        // remove last columns because it has a different pattern (<td colspan=3> </td>)
        columns = getRowColumns(rows[11].replaceAll("<td colspan=3> </td>$", ""));
        duaMap.put("5.1. TERMINAL", columns.get(1));
        duaMap.put("5.1. CODIGO TERMINAL", columns.get(2));

        // Row 13: 2 (5.5. TOTAL PESO NETO (KG)), 4 (5.6. TOTAL PESO BRUTO(KG))
        columns = getRowColumns(rows[12]);
        duaMap.put("5.5 TOTAL PESO NETO (KG)", columns.get(1));
        duaMap.put("5.6 TOTAL PESO BRUTO(KG)", columns.get(3));

        // Row 14: 2 (5.7. TOTAL CANTIDAD BULTOS :), 4 (5.8. TOTAL UNIDADES FISICAS :)
        columns = getRowColumns(rows[13]);
        duaMap.put("5.7 TOTAL CANTIDAD BULTOS", columns.get(1));
        duaMap.put("5.8 TOTAL UNIDADES FISICAS", columns.get(3));

        // Row 15: 2 (5.9. UNIDADES COMERCIALES :), 4 (5.10 RES.EXO.NUM. :)
        columns = getRowColumns(rows[14]);
        duaMap.put("5.9 UNIDADES COMERCIALES", columns.get(1));
        duaMap.put("5.10 RES.EXO.NUM.", columns.get(3));

        // Row 16: 2 (5.12 TOTAL SERIES), 4 (5.13 TIPO TRATAMIENTO)
        columns = getRowColumns(rows[15]);
        duaMap.put("5.12 TOTAL SERIES", columns.get(1));
        duaMap.put("5.13 TIPO TRATAMIENTO", columns.get(3));

        // Row 18: 2 (6.1 FOB MONEDA TRANSACCION), 3 (6.1 FOB MONEDA TOTAL DOLARES)
        columns = getRowColumns(rows[17]);
        duaMap.put("6.1 FOB MONEDA TRANSACCION", columns.get(1));
        duaMap.put("6.1 FOB MONEDA TOTAL DOLARES", columns.get(2));

        // Row 19:  2 (6.2 FLETE MONEDA TRANSACCION), 3 (6.2 FLETE MONEDA TOTAL DOLARES)
        columns = getRowColumns(rows[18]);
        duaMap.put("6.2 FLETE MONEDA TRANSACCION", columns.get(1));
        duaMap.put("6.2 FLETE MONEDA TOTAL DOLARES", columns.get(2));

        // Row 20:  2 (6.4 SEGURO MONEDA TRANSACCION), 3 (6.4 SEGURO MONEDA TOTAL DOLARES)
        columns = getRowColumns(rows[19]);
        duaMap.put("6.4 SEGURO MONEDA TRANSACCION", columns.get(1));
        duaMap.put("6.4 SEGURO MONEDA TOTAL DOLARES", columns.get(2));

        // Row 21:  2 (6.5 CIF MONEDA TRANSACCION), 3 (6.5 CIF MONEDA TOTAL DOLARES)
        columns = getRowColumns(rows[20]);
        duaMap.put("6.5 CIF MONEDA TRANSACCION", columns.get(1));
        duaMap.put("6.5 CIF MONEDA TOTAL DOLARES", columns.get(2));

        // Row 24: 2 (3.1 AD/VALOREM LIQUIDACION), 3 (3.1 AD/VALOREM LIBERACION),
        // 4 (3.1 AD/VALOREM CANTIDAD A PAGAR)
        columns = getRowColumns(rows[23]);
        duaMap.put("3.1 AD/VALOREM LIQUIDACION", columns.get(1));
        duaMap.put("3.1 AD/VALOREM LIBERACION", columns.get(2));
        duaMap.put("3.1 AD/VALOREM CANTIDAD A PAGAR", columns.get(3));

        // Row 25: 2 (3.2 DERECHO ESPECIFICO LIQUIDACION), 3 (3.2 DERECHO ESPECIFICO LIBERACION),
        // 4 (3.2 DERECHO ESPECIFICO CANTIDAD A PAGAR)
        columns = getRowColumns(rows[24]);
        duaMap.put("3.2 DERECHO ESPECIFICO LIQUIDACION", columns.get(1));
        duaMap.put("3.2 DERECHO ESPECIFICO LIBERACION", columns.get(2));
        duaMap.put("3.2 DERECHO ESPECIFICO CANTIDAD A PAGAR", columns.get(3));

        // Row 26: 2 (3.3 IMP. SELECTIVO CONSUMO LIQUIDACION), 3 (3.3 IMP. SELECTIVO CONSUMO LIBERACION),
        // 4 (3.3 IMP. SELECTIVO CONSUMO CANTIDAD A PAGAR)
        columns = getRowColumns(rows[25]);
        duaMap.put("3.3 IMP. SELECTIVO CONSUMO LIQUIDACION", columns.get(1));
        duaMap.put("3.3 IMP. SELECTIVO CONSUMO LIBERACION", columns.get(2));
        duaMap.put("3.3 IMP. SELECTIVO CONSUMO CANTIDAD A PAGAR", columns.get(3));

        // Row 27: 2 (3.4 IMP. PROMOCION MUNICIPAL LIQUIDACION), 3 (3.4 IMP. PROMOCION MUNICIPAL LIBERACION),
        // 4 (3.4 IMP. PROMOCION MUNICIPAL CANTIDAD A PAGAR)
        columns = getRowColumns(rows[26]);
        duaMap.put("3.4 IMP. PROMOCION MUNICIPAL LIQUIDACION", columns.get(1));
        duaMap.put("3.4 IMP. PROMOCION MUNICIPAL LIBERACION", columns.get(2));
        duaMap.put("3.4 IMP. PROMOCION MUNICIPAL CANTIDAD A PAGAR", columns.get(3));

        // Row 28: 2 (3.5 IMP. GENERAL A LA VENTA LIQUIDACION), 3 (3.5 IMP. GENERAL A LA VENTA LIBERACION),
        // 4 (3.5 IMP. GENERAL A LA VENTA CANTIDAD A PAGAR)
        columns = getRowColumns(rows[27]);
        duaMap.put("3.5 IMP. GENERAL A LA VENTA LIQUIDACION", columns.get(1));
        duaMap.put("3.5 IMP. GENERAL A LA VENTA LIBERACION", columns.get(2));
        duaMap.put("3.5 IMP. GENERAL A LA VENTA CANTIDAD A PAGAR", columns.get(3));

        // Row 29: 2 (3.6 DERECHO ANTIDUMPING LIQUIDACION), 3 (3.6 DERECHO ANTIDUMPING LIBERACION),
        // 4 (3.6 DERECHO ANTIDUMPING CANTIDAD A PAGAR)
        columns = getRowColumns(rows[28]);
        duaMap.put("3.6 DERECHO ANTIDUMPING LIQUIDACION", columns.get(1));
        duaMap.put("3.6 DERECHO ANTIDUMPING LIBERACION", columns.get(2));
        duaMap.put("3.6 DERECHO ANTIDUMPING CANTIDAD A PAGAR", columns.get(3));

        // Row 30: 2 (3.7 TASA SERVICIO DESPACHO LIQUIDACION), 3 (3.7 TASA SERVICIO DESPACHO LIBERACION),
        // 4 (3.7 TASA SERVICIO DESPACHO CANTIDAD A PAGAR)
        columns = getRowColumns(rows[29]);
        duaMap.put("3.7 TASA SERVICIO DESPACHO LIQUIDACION", columns.get(1));
        duaMap.put("3.7 TASA SERVICIO DESPACHO LIBERACION", columns.get(2));
        duaMap.put("3.7 TASA SERVICIO DESPACHO CANTIDAD A PAGAR", columns.get(3));

        // Row 31: 2 (3.8 RECARGO NUMERACION LIQUIDACION), 3 (3.8 RECARGO NUMERACION LIBERACION),
        // 4 (3.8 RECARGO NUMERACION CANTIDAD A PAGAR)
        columns = getRowColumns(rows[30]);
        duaMap.put("3.8 RECARGO NUMERACION LIQUIDACION", columns.get(1));
        duaMap.put("3.8 RECARGO NUMERACION LIBERACION", columns.get(2));
        duaMap.put("3.8 RECARGO NUMERACION CANTIDAD A PAGAR", columns.get(3));

        // Row 32: 2 (3.9 SOBRETASA ADICIONAL LIQUIDACION), 3 (3.9 SOBRETASA ADICIONAL LIBERACION),
        // 4 (3.9 SOBRETASA ADICIONAL CANTIDAD A PAGAR)
        columns = getRowColumns(rows[31]);
        duaMap.put("3.9 SOBRETASA ADICIONAL LIQUIDACION", columns.get(1));
        duaMap.put("3.9 SOBRETASA ADICIONAL LIBERACION", columns.get(2));
        duaMap.put("3.9 SOBRETASA ADICIONAL CANTIDAD A PAGAR", columns.get(3));

        // Row 33: 2 (3.11 TOTAL LIQUIDACION)
        columns = getRowColumns(rows[32]);
        duaMap.put("3.11 TOTAL LIQUIDACION", columns.get(1));

        // Row 34: 2 (ULTIMO DIA DE PAGO), 4 (FECHA CANCELACION),
        // 6 (BANCO DE CANCELACION)
        // Last line, remove end of table
        columns = getRowColumns(rows[33].replaceAll("</tr></table><br>.*", ""));
        duaMap.put("ULTIMO DIA DE PAGO", columns.get(1));
        duaMap.put("FECHA CANCELACION", columns.get(3));
        duaMap.put("BANCO DE CANCELACION", columns.get(5));

        return duaMap;
    }

    /**
     * Parse the series values contained in a series block.
     * @param seriesBlock table rows containing a series
     * @param seriesNumber series number
     * @return a map containing column names as keys and series values.
     */
    private static Map<String, String> parseIndivisualSeries(String seriesBlock, int seriesNumber)
    {
        Map<String, String> seriesMap;
        List<String> columns;
        int i;
        StringBuilder descMercancias;

        seriesMap = new TreeMap<String, String>();

        // Replace HTML space codes by spaces
        seriesBlock = seriesBlock.replaceAll("&nbsp;", " ");
        // Split rows in series block
        String[] rows = seriesBlock.split("</tr><tr>");

        seriesMap.put("SERIE", String.valueOf(seriesNumber));
        // Row 1: 1 (PUERTO EMBARQUE), 2 (GUIA AEREA O B/L), 3 (FEC.EMB.),
        //4 (DECL.PREC.), 5 (ITEM), 6 (FEC.VCTO), 7 (CERT.ORIGEN), 8 (ESTADO)
        columns = getRowColumns(rows[0]);
        i = 0;
        seriesMap.put("PUERTO EMBARQUE", columns.get(i++));
        seriesMap.put("GUIA AEREA O B/L", columns.get(i++));
        seriesMap.put("FEC.EMB.", columns.get(i++));
        seriesMap.put("DECL.PREC.", columns.get(i++));
        seriesMap.put("ITEM", columns.get(i++));
        seriesMap.put("FEC.VCTO", columns.get(i++));
        seriesMap.put("CERT.ORIGEN", columns.get(i++));
        seriesMap.put("ESTADO", columns.get(i++));

        // Row 2: 2 (CANT. BULTOS), 3 (CLASE), 4 (UNIDADES FISICAS U.F.),
        // 5 (PESO NETO (KG)), 6 (PESO BRUTO (KG)), 7 (MONEDA TRANSC.), 8 (FOB US $)
        columns = getRowColumns(rows[1]);
        i = 1;
        seriesMap.put("CANT. BULTOS", columns.get(i++));
        seriesMap.put("CLASE", columns.get(i++));
        seriesMap.put("UNIDADES FISICAS U.F.", columns.get(i++));
        seriesMap.put("PESO NETO (KG)", columns.get(i++));
        seriesMap.put("PESO BRUTO (KG)", columns.get(i++));
        seriesMap.put("MONEDA TRANSC.", columns.get(i++));
        seriesMap.put("FOB US $", columns.get(i++));

        // Row 3: 2 (FLETE), 3 (SEGURO), 4 (ADV $), 5 (IGV $), 6 (IPM $), 7 (ISC $)
        columns = getRowColumns(rows[2]);
        i = 1;
        seriesMap.put("FLETE", columns.get(i++));
        seriesMap.put("SEGURO", columns.get(i++));
        seriesMap.put("ADV $", columns.get(i++));
        seriesMap.put("IGV $", columns.get(i++));
        seriesMap.put("IPM $", columns.get(i++));
        seriesMap.put("ISC $", columns.get(i++));

        // Row 4: 2 (PAIS ORIGEN), 3 (PAIS ADQUISICION), 4 (TRATO PREF.INT.),
        // 5 (TRATO PREF. NAC.), 6 (COD. LIBER.)
        columns = getRowColumns(rows[3]);
        i = 1;
        seriesMap.put("PAIS ORIGEN", columns.get(i++));
        seriesMap.put("PAIS ADQUISICION", columns.get(i++));
        seriesMap.put("TRATO PREF.INT.", columns.get(i++));
        seriesMap.put("TRATO PREF. NAC.", columns.get(i++));
        seriesMap.put("COD. LIBER.", columns.get(i++));

        // Row 5: 2 (CERT.INSPECC.1), 3 (CERT.INSPECC.2), 4 (CERT.INSPECC.3),
        // 5 (No. FACTURA 1), 6 (No. FACTURA 2), 7 (No. FACTURA 3)
        columns = getRowColumns(rows[4]);
        i = 1;
        seriesMap.put("CERT.INSPECC.1", columns.get(i++));
        seriesMap.put("CERT.INSPECC.2", columns.get(i++));
        seriesMap.put("CERT.INSPECC.3", columns.get(i++));
        seriesMap.put("No. FACTURA 1", columns.get(i++));
        seriesMap.put("No. FACTURA 2", columns.get(i++));
        seriesMap.put("No. FACTURA 3", columns.get(i++));

        // Row 6: 2 (NANDINA), 3 (DESCRIPCION DE PARTIDA ARANCELARIA),
        // 4 (NABANDINA), 5 (TIPO MRGN), 6 (UNIDADES COMERCIALES), 7 (TIPO U.C.)
        columns = getRowColumns(rows[5]);
        i = 1;
        seriesMap.put("NANDINA", columns.get(i++));
        seriesMap.put("DESCRIPCION DE PARTIDA ARANCELARIA", columns.get(i++));
        seriesMap.put("NABANDINA", columns.get(i++));
        seriesMap.put("TIPO MRGN", columns.get(i++));
        seriesMap.put("UNIDADES COMERCIALES", columns.get(i++));
        seriesMap.put("TIPO U.C.", columns.get(i++));

        // Row 7 - N, Value is in various rows in col 2 (DESCRIPCION DE MERCANCIAS)
        descMercancias = new StringBuilder();
        for (i = 6; i < rows.length; i++)
        {
            // Last row contains a colspan of 9
            if (rows[i].contains("colspan=9"))
            {
                break;
            }
            // Append / to description to separate values
            if (descMercancias.length() > 0)
            {
                descMercancias.append(" / ");
            }
            columns = getRowColumns(rows[i]);
            descMercancias.append(columns.get(1));
        }
        seriesMap.put("DESCRIPCION DE MERCANCIAS", descMercancias.toString());
        return seriesMap;
    }

    /**
     * Split the column contents in a table row.
     * @param seriesRow series row.
     * @return a list containing the column contents of a row.
     */
    private static List<String> getRowColumns(String seriesRow)
    {
        List<String> result;
        String[] cols;

        result = new ArrayList<String>();

        // Remove starting and ending tags
        seriesRow = seriesRow.replaceFirst("^<td( colspan=[0-9])?><font size=1>", "");
        seriesRow = seriesRow.replaceAll("</font></td>$", "");
        cols = seriesRow.split("</font></td><td( colspan=[0-9])?><font size=1>");

        // Remove spaces and add elements to list
        for (String col : cols)
        {
            result.add(col.trim());
        }
        return result;
    }

    /**
     * Find the number of series in the DUA report.
     * @param page HTML containing the DUA report.
     * @return -1 if the pattern is not found.
     */
    private static int parseSeriesCount(String page)
    {
        String regEx;
        Pattern pattern;
        Matcher matcher;
        int seriesCount = -1;

        // Find the number of series in the following pattern:
        // 5.12 TOTAL SERIES          : </font></td><td><font size=1>332</font>
        regEx = "(5\\.12 TOTAL SERIES          : </font></td><td><font size=1>)([0-9]{1,})(</font>)";
        pattern = Pattern.compile(regEx);
        matcher = pattern.matcher(page);
        if (matcher.find())
        {
            seriesCount = Integer.parseInt(matcher.group(2));
        }
        return seriesCount;
    }

    /**
     * Get all the import links in a set of customs links given by the search results.
     * @param customsLinks one link for every customs in the search results.
     * @return a list of import beans containing the link to the DUA.
     * @throws java.lang.Exception in case of error.
     */
    public List<ImportBean> readCountryImportLinks(List<String> customsLinks) throws Exception
    {
        List<ImportBean> importBeans;
        List<String> links;
        String linksPage;
        int i = 1;

        importBeans = new LinkedList<ImportBean>();
        // For each customs link, get all the links and add them to the results
        for (String customsLink : customsLinks)
        {
            log.info("Processing customs " + (i++) + " of " + customsLinks.size());
            // Get page
            linksPage = this.downloader.doGet(customsLink);
            // Parse links
            links = parseImportLinks(linksPage);
            log.info("Found " + links.size() + " links");
            // Add links to beans
            for (String importLink : links)
            {
                importBeans.add(new ImportBean(importLink));
            }
        }
        log.info("Found " + importBeans.size() + " import links");
        return importBeans;
    }

    /**
     * Do a search for every customs code to find the countries that have
     * imports. The search is based on the start and end dates provided in the
     * constructor.
     * @return a map containing country codes as keys and all corresponding
     * links for each custom with results.
     * @throws java.lang.Exception in case of error
     */
    public Map<String, List<String>> readCountryCustomsLinks() throws Exception
    {
        Map<String, String> links;
        Map<String, List<String>> results;
        String searchResults;
        String url;
        String params;

        results = new TreeMap<String, List<String>>();
        url = "http://www.aduanet.gob.pe/servlet/SGDec10A";

        // Request links for each customs
        for (String custom : CUSTOMS)
        {
            log.info("Reading customs " + custom);

            params = "orden=CIF&FecInicial=" + startDate + "&FecFinal=" +
                    endDate + "&pdato=" + custom + "&dona=01&Select=paisori&Codigo=&opc=4";
            searchResults = downloader.doPost(url, params);

            // get links from page
            links = parseCountryCustomsLinks(searchResults);
            log.debug("Found " + links.size() + " in custom " + custom);

            // Add links to results
            for (String country : links.keySet())
            {
                if (results.containsKey(country) == false)
                {
                    results.put(country, new ArrayList<String>());
                }
                results.get(country).add(links.get(country));
            }
        }

        return results;
    }

    /**
     * Parse the links with the country codes from the search by customs results.
     * @param page results from customs search.
     * @return a map containing country codes as keys and links as values.
     * @throws java.lang.Exception when there is an error.
     */
    public static Map<String, String> parseCountryCustomsLinks(String page)
    {
        Map<String, String> countryLinksMap;
        String link;
        String country;
        String regEx;
        Pattern pattern;
        Matcher matcher;

        countryLinksMap = new TreeMap<String, String>();

        // Find the following pattern:
        // <a href="/servlet/SgDetUniAgenteA?codigo=LK&fini=20050101&ffin=20050131&
        // cagente=118&opcion=paisori&tipo=4&serv=4"> <strong>LK</strong>
        // </a>SRI LANKA</font></td>
        regEx = "(/servlet/SgDetUniAgenteA\\?[a-zA-z0-9&= ]*)(.{0,10}strong>)([a-zA-z0-9]{2})";
        pattern = Pattern.compile(regEx);
        matcher = pattern.matcher(page);
        while (matcher.find())
        {
            // Add each link to the results map
            link = SERVER_URL + matcher.group(1);
            country = matcher.group(3);
            log.debug(link + " " + country);
            countryLinksMap.put(country, link);
        }
        log.debug("Found " + countryLinksMap.size() + " links");

        // Check for errors, if there are no links then the following message
        // must be present: "No existen registros"
        if (countryLinksMap.size() == 0)
        {
            if (page.contains("No existen registros") == false)
            {
                log.error("Couldn't find links in page " + page);
                throw new RuntimeException("Couldn't find links in page");
            }
        }

        return countryLinksMap;
    }

    /**
     * Get all the import links given a search results page. The links point to
     * the import data.
     * @param page HTML page containing search results.
     * @return a list of links found in the page.
     * @throws java.lang.RuntimeException when the page contains no links.
     */
    public static List<String> parseImportLinks(String page)
    {
        List<String> links;
        List<String> returnedLinks;
        String regEx;

        // Find the following pattern:
        // <a href="/servlet/SgCDUI2?codaduana=118&numecorre=000001&anoprese=2005
        // &option=una&n=10" target = '_top'>
        regEx = "/servlet/SgCDUI2\\?[a-zA-z0-9&= ]*";
        links = findPatterns(page, regEx, 0);

        // Test if links were found. There must be at least one link
        if (links.size() == 0)
        {
            log.error("Did not find import links in page " + page);
            throw new RuntimeException("Did not find import links");
        }

        // Add server to every link
        returnedLinks = new ArrayList<String>();
        for (String link : links)
        {
            // Add each link to the results list
            link = SERVER_URL + link;
            returnedLinks.add(link);
            log.debug(link);
        }
        log.debug("Found " + returnedLinks.size() + " links");

        return returnedLinks;
    }
}
